package com.ysq.ip.spider;

import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.ysq.ip.spider.config.BaseConfigInfo;
import com.ysq.ip.spider.util.CommonUtil;

/**
 * 
 * @author admin
 *
 */
public class IpSpider {
	static Logger logger = CommonUtil.getLogger(IpSpider.class);
	
	/** 文件输出路径 **/
	String outputPath;
	
	/** 文件输出流池 **/
	Map<String, Writer> outWriterMap;
	
	public IpSpider(){
		outputPath = BaseConfigInfo.projectRootDir + "/output/";
		
		outWriterMap = new HashMap<String, Writer>();
	}
	
	
	/**
	 * 获取 ip 地址段
	 */
	public String getIpPool(Element partEle){
		String ipPool = "";
		String partHtml = partEle.outerHtml();
		
		String reg = "<p\\s*>[^<>]+<\\s*br\\s*/?>";
		
		Pattern pattern = Pattern.compile(reg);
		Matcher match = pattern.matcher(partHtml);
		
		if(match.find()){
			ipPool = match.group();
		}
		
		ipPool = ipPool.replaceAll("<[^<>]+>", "");
		
		return ipPool;
	}
	
	/**
	 * 获取区域段
	 * @param partEle
	 * @return
	 */
	public String getAreaPart(Element partEle){
		String result = "";
		
		String partHtml = partEle.outerHtml();
		
		//(?=exp)	匹配exp前面的位置
		//(?<=exp)	匹配exp后面的位置
		//零宽断言	不正确
		//String reg = "(?<=\\<\\s*br\\s*/?\\>).*(?=\\<\\s*br\\s*/?\\>)";
		
		String reg = "<\\s*br\\s*/?>.*<\\s*br\\s*/?>";
		
		Pattern pattern = Pattern.compile(reg);
		Matcher match = pattern.matcher(partHtml);
		if(match.find()){
			result = match.group();
		}
		
		result = result.replaceAll("<\\s*br\\s*/?>", "");
		
		return result;
	}
	
	/**
	 * 获取网页
	 * @param url
	 * @return
	 * @throws Exception
	 */
	public Document spiderHtml(String url) throws Exception{
		Document doc = Jsoup.connect(url).timeout(40000).get();
		
		//CommonUtil.writeFileContent(path, doc.outerHtml(), false);
		
		return doc;
	}
	
	/**
	 * 测试用，根据文件加载
	 * @param path
	 * @return
	 * @throws Exception
	 */
	public Document spiderPath(String path) throws Exception{
		String htmlContent = CommonUtil.getFileContent(path);
		
		Document doc = Jsoup.parse(htmlContent);
		
		return doc;
	}
	
	/**
	 * 获得详情页面ip
	 * @param doc
	 * @return
	 */
	public String getDetailIpContent(Document doc){
		
		Element ipEle = doc.select("[class=ipcontent] [class=iplist] p").get(1);
		
		String ipContent = ipEle.text();
		
		return ipContent;
	}
	
	/**
	 * ip 地址少，没有 detail 的情况
	 * @param partEle
	 * @return
	 */
	public String getLittleIpContent(Element partEle){
		String ipContent = "";
		String partHtml = partEle.outerHtml();
		
		String reg = "<\\s*br\\s*/?>[^<>]+</p\\s*>";
		
		Pattern pattern = Pattern.compile(reg);
		Matcher match = pattern.matcher(partHtml);
		
		if(match.find()){
			ipContent = match.group();
		}
		
		ipContent = ipContent.replaceAll("<[^<>]+>", "");
		
		return ipContent;
	}
	
	/**
	 * 解析 ip 详情，并写入文件
	 * @param partEle
	 */
	public String getIpDetail(Element partEle) throws Exception{
		Element aEle = partEle.select("a").first();
		
		String ipContent = null;
		
		if(aEle != null){//包含查看 detail
		
			String detailUrl = aEle.attr("abs:href");
			
			Document doc = spiderHtml(detailUrl);
			
			ipContent = getDetailIpContent(doc);
			
		}else{//不包含
			ipContent = getLittleIpContent(partEle);
		}
		
		return ipContent;
	}
	
	/**
	 * 获得文件绝对路径，并创建路径文件夹
	 * @param filePath
	 * @return
	 */
	public String getFilePath(String filePath){
		
		File file = new File(filePath);
		
		File parentFile = file.getParentFile();
		
		if(!parentFile.exists()){
			parentFile.mkdirs();
		}
		
		String resultPaht = file.getAbsolutePath();
		
		return resultPaht;
	}
	
	/**
	 * 通过 area 获得 writer
	 * @param areaName
	 * @return
	 */
	public Writer getIpWriter(String areaName) throws Exception{
		
		Writer ipWriter = null;
		if(outWriterMap.containsKey(areaName)){
			
			ipWriter = outWriterMap.get(areaName);
		}else{
			
			String ipPath = outputPath + areaName + ".txt";
			
			ipPath = getFilePath(ipPath);
			
			OutputStream output = new FileOutputStream(ipPath);
			
			ipWriter = new OutputStreamWriter(output);
			
			outWriterMap.put(areaName, ipWriter);
		}
		
		return ipWriter;
	}
	
	/**
	 * 将 Ip 写入文件
	 * @param areaName
	 * @param ipDetail
	 */
	public int writeIpContent(String areaName, String ipDetail){
		int ipCount = 0;
		try {
			Writer ipWriter = getIpWriter(areaName);
			
			String[] ipArray = ipDetail.split(" ");
			ipCount = ipArray.length;
			for(String oneIp : ipArray){
				ipWriter.write(oneIp + "\r\n");
				
				ipWriter.flush();
			}
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return ipCount;
	}
	
	/**
	 * 根据地域名称，抓取ip
	 */
	public void spiderAreaIp(String ipUrl) throws Exception{
		int ipCount = 0;
		
		Document doc = spiderHtml(ipUrl);
		//Document doc = spiderPath(toPath);
		
		Elements selEles = doc.select("[class=ipcontent] [class=iplist] p");
		
		for(Element partEle : selEles){
			
			String areaPart = getAreaPart(partEle);
			
			for(String oneArea : BaseConfigInfo.area_list){
				
				if(areaPart.indexOf(oneArea) > -1){
					
					String ipPool = getIpPool(partEle);
					
					String ipDetail = getIpDetail(partEle);
					
					int partCount = writeIpContent(oneArea, ipDetail);
					
					logger.info("ip count : " + partCount + " | area : " + areaPart + " | ip : " + ipPool + " write complete ... ");
					
					ipCount += partCount;
					
					break;
				}
			}
		}
		logger.info("url : " + ipUrl + " complete. total ip : " + ipCount);
	}
	
	/**
	 * 爬取每一个 url
	 */
	public void startSpider(){
		
		//String enterUrl = "http://www.580san.com/ip.php?";
		
		
		for(String ipUrl : BaseConfigInfo.url_list){
			
			try {
				spiderAreaIp(ipUrl);
			} catch (Exception e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			
		}
		
	}
	
	public static void main(String[] args) throws Exception {
		IpSpider is = new IpSpider();
		
		is.startSpider();
		
		System.out.println("finished ... ");
	}
}
